######## Yahoo Search Client ########
######## Final Year Project 2011#####
######## Making Web Search Spatial###


######## Liam Sullivan       ########
######## University of Leeds ########

######## Based on Tutorials found at:

######## http://www.42topics.com/dumps/appengine-search/doc.html

######## http://code.google.com/appengine/docs/python/gettingstarted/



import wsgiref.handlers
import urllib
import logging
import time

from google.appengine.ext import webapp
from google.appengine.ext.webapp import template
from google.appengine.api import urlfetch

from google.appengine.ext import db
from google.appengine.api import users

from django.utils import simplejson
from operator import itemgetter
from StringIO import StringIO


## Import customfilters.py, this sits in the top level directory and contains customer filters for use within the template language

webapp.template.register_template_library('customfilters')



class Website(db.Model):
    """A class for storing classification data for a URL in the google appengine filestore"""
    url = db.StringProperty(required=True)
    Arts = db.StringProperty(required=True)
    Business = db.StringProperty(required=True)
    Science = db.StringProperty(required=True)
    Computers = db.StringProperty(required=True)
    Recreation = db.StringProperty(required=True)
    Sports = db.StringProperty(required=True)
    Society = db.StringProperty(required=True)
    Health = db.StringProperty(required=True)
    Home = db.StringProperty(required=True)
    Games = db.StringProperty(required=True)


def DebugPrint(item):
        
    print "Content-Type: text/html"
    print "<h2> " + item + " </h2>"
    
class SearchResult():
    """SearchResult objects have a Title, Extract, URL and Classification data"""
    
    def __init__(self, URL, ClickURL, Title, Extract, relevance):
        """Build object with passed in values but also generate Classification data from URL and encode in Object"""
        self.URL = URL
        self.ClickURL = ClickURL
        self.Title = Title
        self.Extract = Extract
        
        self.relevance = relevance
               
        #Get classification data
        self.Classification = getClassificationData(ClickURL)
        
        TopClass = ([0,"Nothing"])
        
        #Multiply classification values by 100 to get probabilities as %
        for key, value in self.Classification.iteritems():
            
            
            logging.info("It was: ")
            logging.info(value)
            NewValue = value * 100
            
            logging.info("Its now: ")
            logging.info(NewValue)
            
            NewValue = int(NewValue)
            
            logging.info("Finally after rounding it is: ")
            logging.info(NewValue)
           
            if NewValue > TopClass[0]:
                self.TopClass = ([NewValue, key])
                
                
            
            #And alter the value for the key
            self.Classification[key] = NewValue
            
        self.TopClass = self.TopClass
            
        self.Arts = self.Classification['Arts']
        self.Business = self.Classification['Business']
        self.Science = self.Classification['Science']
        self.Computers = self.Classification['Computers']
        self.Recreation = self.Classification['Recreation']
        self.Sports = self.Classification['Sports']
        self.Society = self.Classification['Society']
        self.Health = self.Classification['Health']
        self.Home = self.Classification['Home']  
        self.Games = self.Classification['Games'] 
       
class MainPage(webapp.RequestHandler):
    def get(self):
        
        MainError = True
        
        #Write the header out
        
        self.response.headers['Content-Type'] = 'text/html'
        
        #Get the query from the text box which is passed in through the RequestHandler
        query = self.request.get('q', '')
        logging.info("Query is:  " + query)
        
        
        #Get the value of the radio set for command choice
        radio = self.request.get('radio', '')         
        logging.info("Radio selection is :  " + radio)        
     
        
        #Get number of results and int it
        
        NumberOfResults = self.request.get('number', '')
        
        #If there is already a query string then we'll print result        

        
        if radio == "traditional":
            
          """ Present a classification map for for all results, showing distance from topics"""
          
          # Flag off to prevent error page
          MainError = False
          
          
          #Get number of results and int it
          
          logging.info("Should do traditional")
          
          NumberOfResults = self.request.get('number', '')
          
          logging.info("Number of results requested: " + NumberOfResults)
          
          logging.debug('Query is : %s'% query)
         # results = get_search_results('Ix4800TV34H7Rh8ggZJ0Nk.FRU3RDM3w5CEOhPsBL5yQCOqpnKdFcU_nD_twMo_7_oa5kmqDtbfl0eybWwOtjZqbM7ahsM4-', query,'','',NumberOfResults)
          #results = results['Result']
          
          #new method
          appid = 'Ix4800TV34H7Rh8ggZJ0Nk.FRU3RDM3w5CEOhPsBL5yQCOqpnKdFcU_nD_twMo_7_oa5kmqDtbfl0eybWwOtjZqbM7ahsM4-'
          
          ResultObjectsList = getTraditionalSearchResults(appid, query, NumberOfResults, region ='us', type = 'all', start = 0, format ='any', adult_ok = "", similar_ok = "", language = "", country = "", site = "", subscription = "", license = '')
          
          Traditional = True
                    
          resp = template.render('classicsearch.html', locals())     
         
        if radio == "classify":
            
          """ Present a classification map for for all results, showing distance from topics"""
          
          # Flag off to prevent error page
          MainError = False
          
          
          #Get number of results and int it
          
          NumberOfResults = self.request.get('number', '')
          
          logging.info("Number of results requested: " + NumberOfResults)
          
          logging.debug('Query is : %s'% query)
         # results = get_search_results('Ix4800TV34H7Rh8ggZJ0Nk.FRU3RDM3w5CEOhPsBL5yQCOqpnKdFcU_nD_twMo_7_oa5kmqDtbfl0eybWwOtjZqbM7ahsM4-', query,'','',NumberOfResults)
          #results = results['Result']
          
          #new method
          appid = 'Ix4800TV34H7Rh8ggZJ0Nk.FRU3RDM3w5CEOhPsBL5yQCOqpnKdFcU_nD_twMo_7_oa5kmqDtbfl0eybWwOtjZqbM7ahsM4-'
          
          ResultObjectsList = getSearchResults(appid, query, NumberOfResults, region ='us', type = 'all', start = 0, format ='any', adult_ok = "", similar_ok = "", language = "", country = "", site = "", subscription = "", license = '')
          
          Classify = True
          
          resp = template.render('classify.html', locals())
        
        if radio == "spatial":
            
            """ Present a classification map for for all results, showing distance from topics"""
            
            # Flag off to prevent error page
            MainError = False
                    
            #Get number of results and int it
            
            NumberOfResults = self.request.get('number', '')
            
            logging.info("Number of results requested: " + NumberOfResults)
            
            logging.debug('Query is : %s'% query)
            
            appid = 'Ix4800TV34H7Rh8ggZJ0Nk.FRU3RDM3w5CEOhPsBL5yQCOqpnKdFcU_nD_twMo_7_oa5kmqDtbfl0eybWwOtjZqbM7ahsM4-'
            
            ResultObjectsList = getSearchResults(appid, query, NumberOfResults, region ='us', type = 'all', start = 0, format ='any', adult_ok = "", similar_ok = "", language = "", country = "", site = "", subscription = "", license = '')
            
            Classify = True
            
            
            def getTopicCounts(ResultObjectsList):
                """Takes a list of result objects and returns a dictionary with keys as the topics and values giving the number of results classified as that topic, in this initial view this is based on the largest (top) class from each result"""
                output = {}
                
                for res in ResultObjectsList:
                    pass                
                    
                    
                    
            TopicCounts = getTopicCounts(ResultObjectsList)
            
            #Get info about the results list
            
            
            
            #Encode what we find into the JSON to drive the chart
            
            
            DjangoData = {
    "children": [
       {
         "children": [
          
    
         ], 
         "data": {
           "description": "B", 
           "$color": "#B0AAF6", 
           "days": 2, 
           "$angularWidth": 1000, 
           "size": 13880
         }, 
         "id": "Topics/Business", 
         "name": "Business"
       }, 
       {
         "children": [
          
          
         ], 
         "data": {
           "description": "B", 
           "$color": "#3366CC", 
           "days": 3, 
           "$angularWidth": 1000, 
           "size": 35549
         }, 
         "id": "Topics/Computers", 
         "name": "Computers"
       }, 
       {
         "children": [
          
         ], 
         "data": {
           "description": "B", 
           "$color": "#B0AAF6", 
           "days": 50, 
           "$angularWidth": 17000, 
           "size": 50
         }, 
         "id": "Topics/Games", 
         "name": "Games"
       }, 
       {
         "children": [
          
          
         ], 
         "data": {
           "description": "B", 
           "$color": "#3366CC", 
           "days": 3, 
           "$angularWidth": 7000, 
           "size": 87820
         }, 
         "id": "Topics/Health", 
         "name": "Health"
       }, 
       {
         "children": [
          
          
          
         ], 
         "data": {
           "description": "B", 
           "$color": "#B2ABF4", 
           "days": 3, 
           "$angularWidth": 6000, 
           "size": 32900
         }, 
         "id": "Topics/Home", 
         "name": "Home"
       }, 
       {
         "children": [
          
         ], 
         "data": {
           "description": "B", 
           "$color": "#3366CC", 
           "days": 26, 
           "$angularWidth": 2000, 
           "size": 8079
         }, 
         "id": "Topics/Recreation", 
         "name": "Recreation"
       }, 
       {
         "children": [
                  
      
          
         ], 
         "data": {
           "description": "B", 
           "$color": "#AEA9F8", 
           "days": 1, 
           "$angularWidth": 100, 
           "size": 13575
         }, 
         "id": "Topics/Science", 
         "name": "Scienceeee"
       }, 
       {
         "children": [
          
         ], 
         "data": {
           "description": "B", 
           "$color": "#3366CC", 
           "days": 1, 
           "$angularWidth": 100, 
           "size": 186221
         }, 
         "id": "Topics/Arts", 
         "name": "Arts"
       }
     ], 
     "data": {
       "$type": "none"
     }, 
     "id": "Topics", 
     "name": "All Topics"
   }

            
            resp = template.render('SpatialSearch.html', locals())
            
        if radio == "naivespatial":
            
            """Naive presentation of classifier data"""
            
            # Flag off to prevent error page
            MainError = False
                    
            #Get number of results and int it
            
            NumberOfResults = self.request.get('number', '')
            
            logging.info("Number of results requested: " + NumberOfResults)
            
            logging.debug('Query is : %s'% query)
            
            appid = 'Ix4800TV34H7Rh8ggZJ0Nk.FRU3RDM3w5CEOhPsBL5yQCOqpnKdFcU_nD_twMo_7_oa5kmqDtbfl0eybWwOtjZqbM7ahsM4-'
            
            ResultObjectsList = getSearchResults(appid, query, NumberOfResults, region ='us', type = 'all', start = 0, format ='any', adult_ok = "", similar_ok = "", language = "", country = "", site = "", subscription = "", license = '')
            
            Classify = True
            
            resp = template.render('NaiveSpatialSearch.html', locals())
            
        if radio == "offline":
            """A branch to allow working when the classifier is down"""
            
            # Flag off to prevent error page
            MainError = False
            
            logging.info("Problems with an external source so going to offline / testbed / error page")
            
            resp = template.render('offline.html', locals())
            
            
        #Final case is an error
            
        if MainError:
            logging.info("Either landed on form for first time or something went wrong")
            resp = template.render('form.html', locals())
            
        #Regardles of mode, we respond with something to drive page creation  
        self.response.out.write(resp)


def main():
  application = webapp.WSGIApplication(
                                       [('/', MainPage)],
                                       debug=True)
  wsgiref.handlers.CGIHandler().run(application)



def get_search_results(appid, query, region ='us', type = 'all', results = 2, start = 0, format ='any', adult_ok = "", similar_ok = "", language = "", country = "", site = "", subscription = "", license = ''):
    base_url = u'http://search.yahooapis.com/WebSearchService/V1/webSearch?'
    params = locals()
    result = _query_yahoo(base_url, params)
    return result['ResultSet']

def _query_yahoo(base_url, params):
    params['output'] = 'json'
    payload = urllib.urlencode(params)
    url = base_url + payload
    response = StringIO(urlfetch.fetch(url).content)
    result = simplejson.load(response)
    return result


def getTraditionalSearchResults(appid, query, results, region ='us', type = 'all', start = 0, format ='any', adult_ok = "", similar_ok = "", language = "", country = "", site = "", subscription = "", license = ''):
    """Get search results, call out to classify each one and build a SearchResult object for each, return a list of these"""
   
    base_url = u'http://search.yahooapis.com/WebSearchService/V1/webSearch?'
    params = locals()
    params['output'] = 'json'
    payload = urllib.urlencode(params)
    url = base_url + payload
    response = StringIO(urlfetch.fetch(url).content)
    logging.info("Calling service with URL: ")
    logging.info(url)
    result = simplejson.load(response)
    result = result['ResultSet']
    AllResults = result['Result']
    
    ResultObjectList = []
    
    ResultRank = 1
    
    for res in AllResults:
        """Loop through all results, they are in order of relevance"""
        
        #Don't do any classification as this is just traditional search
        
        #Add it to the list
        ResultObjectList.append(res)
    
    return ResultObjectList
    
    
def getSearchResults(appid, query, results, region ='us', type = 'all', start = 0, format ='any', adult_ok = "", similar_ok = "", language = "", country = "", site = "", subscription = "", license = ''):
    """Get search results, call out to classify each one and build a SearchResult object for each, return a list of these"""
   
    base_url = u'http://search.yahooapis.com/WebSearchService/V1/webSearch?'
    params = locals()
    params['output'] = 'json'
    payload = urllib.urlencode(params)
    url = base_url + payload
    response = StringIO(urlfetch.fetch(url).content)
    logging.info("Calling service with URL: ")
    logging.info(url)
    result = simplejson.load(response)
    result = result['ResultSet']
    AllResults = result['Result']
    
    ResultObjectList = []
    
    ResultRank = 1
    
    for res in AllResults:
        """Loop through all results, they are in order of relevance"""
        
        logging.info(res)
        
        #Make a new SearchResult Object
        SearchResultInstance = SearchResult(res['Url'], res['ClickUrl'], res['Title'], res['Summary'], ResultRank)
        
        #Increment ResultRank
        ResultRank = ResultRank + 1
        
        #Use the object method to fetch classification data before saving object
        
        
        #Add it to the list
        ResultObjectList.append(SearchResultInstance)
    
    return ResultObjectList


def getClassificationData(URL):
    """Classifies a URL using the UClassify Service"""
    
    #Dictionary containing all params, which we will encode for REST Call    
    Params = {}   
        
    #BaseURL
    #uClassify URL Endpoint
    endpoint = "http://uclassify.com/browse/uClassify/Topics/ClassifyUrl?"
    
    #API Key    
    Params['readkey'] = "TsE3A35FCBSAdbGFLEvI2PXH0"
    
    #Output Format
    Params['output'] = "json"
    
    #Incoming URL that needs classifying
    Params['url'] = URL
    
    #Version
    Params['version'] = 1.01
    
    #Strip HTML
    Params['removeHtml'] = True
    
    #Encode Params from dictionary for call
    
    RestParams = urllib.urlencode(Params)    
    
    #Combine base url and params
    
    RestURL = endpoint + RestParams
    
    logging.info("Trying to classify URL : ")
    logging.info(URL)
    
    try:
        #First attempts
        logging.info("First attempt.....")
        response = StringIO(urlfetch.fetch(RestURL, deadline=10).content)   
        logging.info("First attempt worked, got response......") 
        
    except:
        logging.info("First attempt failed")
        time.sleep(1)
        
    try:
        #2nd attempt
        logging.info("2nd attempt.....")
        response = StringIO(urlfetch.fetch(RestURL, deadline=10).content)   
        logging.info("2nd worked, got response......")
        
    except:
        logging.info("2nd attempt failed")
    
    result = simplejson.load(response)
    
    try:
        ResultSet =  result['cls1'] 
        logging.info("Results from classifier: ")
        logging.info(ResultSet)
        return ResultSet
    except:
        logging.info("Didn't get any classifier results")
    

    
    


def ClassifyURL(URL):
    
    """Classifies a URL, calls other method to get the results from the Web Service then computes highest probability and returns as a string"""

    #Pass URL and get the dictionary results from the classifier
    Results = getClassificationData(URL)
    
    #Take the first key from a sorted set of the dictionary, this is the most likely topic as classified
    
    SortedResults = sorted(Results.items(), key=itemgetter(1), reverse=True)
    
    TopResult = SortedResults[0]
    
    return TopResult[0]




if __name__ == "__main__":
    main()
